package crawler.proxy.uitl;

import crawler.douban.DouBanSpider;
import crawler.http.HttpClientTest;
import crawler.proxy.Proxy;
import crawler.proxy.ProxySpider;
import crawler.proxy.impl.*;
import org.apache.log4j.Logger;

import java.util.HashSet;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;

/**
 * USER: lintc 【lintiancong@zhuojianchina.com】
 * DATE: 2017-09-14 14:06
 */
public class ProxyUtil implements Runnable{

    private static final Logger LOG = Logger.getLogger(ProxyUtil.class);

    private ProxySpider proxySpider;

    // 未校验的代理
    private Queue<Proxy> unCheckProxySet = new ConcurrentLinkedQueue<>();

    // 默认代理数量
    private Integer proxyCount = 30;

    private boolean runFlag = true;

    public ProxyUtil(ProxySpider proxySpider, Integer proxyCount) {
        this.proxySpider = proxySpider;
        if (null != proxyCount) {
            this.proxySpider = proxySpider;
        }
    }

    // 停止抓取
    public void stop() {
        this.runFlag = false;
        LOG.info(proxySpider.getCatchUrl() + " : 停止抓取");
    }

    @Override
    public void run() {
        int i = 0;
        Set<Proxy> set = new HashSet<>();
        while (runFlag) {
            if (i > 60 && set.size() == 0) { // 重置页码
                i = 0;
            }
            i++;
            LOG.info("代理IP抓取: (次数)" + i + " : （未校验代理）" + unCheckProxySet.size() + " : （代理）" + ProxyList.proxyIpList.size());
            // 1 获取代理
            set = proxySpider.list(i);
            unCheckProxySet.addAll(proxySpider.list(i));
            // 2 校验代理
            new Thread(() -> {
                check();
            }).start();
            LOG.info("代理IP抓取结束：(次数)" + i + " : （未校验代理）" + unCheckProxySet.size() + " : （代理）" + ProxyList.proxyIpList.size());
            while (unCheckProxySet.size() > proxyCount) {
                try {
                    Thread.sleep(5 * 1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
        LOG.info(proxySpider.getCatchUrl() + " : 抓取退出" + ":(次数)" + i);
    }

    public Proxy getUnCheckProxy() {
        Proxy proxy = unCheckProxySet.poll();
        while (null == proxy) {
            proxy = unCheckProxySet.poll();
            DouBanSpider.friendlyToDouban(1);
        }
        return proxy;
    }

    // 校验代理并入库
    public void check() {
        while (unCheckProxySet.size() > 0) {
            Proxy proxy = getUnCheckProxy();
            if (HttpClientTest.proxy(proxy, HttpClientTest.URL)) {
                ProxyList.setProxy(proxy);
                LOG.info(proxySpider.getCatchUrl() + " : 校验成功 " + proxy + " : （代理）" + ProxyList.proxyIpList.size());
            }
        }
    }

    public static void main(String[] args) {
        new Thread(new ProxyUtil(new JiSuProxy(), null)).start();
//        new Thread(new ProxyUtil(new KuaiProxy(), null)).start();
        new Thread(new ProxyUtil(new MimiProxy(), null)).start();
        new Thread(new ProxyUtil(new OneEightOneProxy(), null)).start();
        new Thread(new ProxyUtil(new SixSixProxy(), null)).start();
        new Thread(new ProxyUtil(new XiciProxy(), null)).start();
    }
}
